package com.chance.cc.crawler.development.bootstrap.toutiao;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.Downloader;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.downloader.http.HttpClientDownloader;
import com.chance.cc.crawler.core.downloader.http.HttpClientGenerator;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.downloader.proxy.SiteConfigProxyProvider;
import com.chance.cc.crawler.core.filter.FilterInfo;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.StringUtils;
import org.junit.Test;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.*;

/**
 * @author lt
 * @version 1.0
 * @date 2021-04-07 11:04:27
 * @email okprog@sina.com
 */
public class TouTiaoStart {
    public static final String domain = "toutiao";
    public static final String site = "searchkw";
    public static final String kwSite = "search_keyword";
    public static final String siteBiz = "news-release";
    private static final String IS_FILTER_ARTICLE = "is_filter_article";

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        //HL89Q19E86E2987D
        //71F33D94CE5F7BF2
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }

    public static void main(String[] args) {

        CrawlerRequestRecord searchKWRecord = searchKWRecord();
        CrawlerRequestRecord supportRecord = getKwRecord(kwSite);
        CrawlerRequestRecord flushNewsRecord = flushNewsRecord();

        //添加评论去重信息
        CrawlerRecord filterCrawlerRecord = new CrawlerRecord();
        filterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.dateRange);
        filterCrawlerRecord.addFilterInfo(FilterUtils.memoryFilterKeyInfo(StringUtils.joinWith("-",filter,domain,"comment")));
        filterCrawlerRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(24,null));
        flushNewsRecord.tagsCreator().bizTags().addCustomKV("comment_record_filter_info", JSON.toJSONString(filterCrawlerRecord));

        DevCrawlerController devCrawlerController = DevCrawlerController.builder()
                .triggerInfo(domain, domain + "_trigger", System.currentTimeMillis(), domain + "_job")
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain)) //内存队列
                .consoleResultPipeline() //控制台输t出
//                .fileResultPipeline("D:\\chance\\data\\toutiao\\toutiao_bk_test.json", false)
                .requestRecord(flushNewsRecord)  //more job
                .supportRecord(userAgentRecord())
//                .requestRecord(searchKWRecord)  //more job
//                .supportRecord(supportRecord)
                .crawlerThreadNum(5)
                .build();


        //是否采集评论
        devCrawlerController.getCrawlerJob().getScheduleTags().getCategoryTag().addLabelTag(CrawlerEnum.CrawlerDataType.comment.enumVal());
        devCrawlerController.getCrawlerJob().getScheduleTags().getCategoryTag().addKVTag(IS_FILTER_ARTICLE,true);
        devCrawlerController.start();


//        articleRecordTest();
//        searchKWRecordTest();
    }

    public static CrawlerRequestRecord searchKWRecord(){
        FilterInfo turnPageFilterInfo = new FilterInfo();
        turnPageFilterInfo.setFilter(CrawlerEnum.CrawlerRecordFilter.count);
        turnPageFilterInfo.setCountTotalNum(100);
        turnPageFilterInfo.setCurCount(0);

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, turnPageItem)
                .httpUrl("https://www.toutiao.com/")
                .releaseTime(System.currentTimeMillis())
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(7*24,null))//最大回溯时间范围
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(StringUtils.joinWith("-","filter",domain,site,"queue")))
                .turnPageFilterInfo(turnPageFilterInfo)
                .needWashed(false)
                .needParsed(false)
//                .proxy(proxy)
                .build();
        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);
        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        requestRecord.tagsCreator().bizTags().addSiteBiz(siteBiz);
        return requestRecord;
    }

    public static void searchKWRecordTest(){
        String url  = "https://so.toutiao.com/search?keyword=nike&pd=information&source=search_subtab_switch&dvpf=pc&aid=4916&page_num=0";
//        https://so.toutiao.com/search?keyword=nike&pd=information&source=search_subtab_switch&dvpf=pc&aid=4916&page_num=0
        FilterInfo turnPageFilterInfo = new FilterInfo();
        turnPageFilterInfo.setFilter(CrawlerEnum.CrawlerRecordFilter.count);
        turnPageFilterInfo.setCountTotalNum(100);
        turnPageFilterInfo.setCurCount(0);
        HttpConfig httpConfig = HttpConfig.me(domain);
        httpConfig.setHttpSite("test");
        httpConfig.setDisableCookie(true);

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, turnPage)
                .httpUrl(url)
                .httpConfig(httpConfig)
                .releaseTime(System.currentTimeMillis())
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(7*24,null))//最大回溯时间范围
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(StringUtils.joinWith("-","filter",domain,site,"queue")))
                .turnPageFilterInfo(turnPageFilterInfo)
//                .proxy(proxy)
                .build();
        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        requestRecord.tagsCreator().bizTags().addSiteBiz(siteBiz);


        //添加评论去重信息
        CrawlerRecord filterCrawlerRecord = new CrawlerRecord();
        filterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.key);
        filterCrawlerRecord.addFilterInfo(FilterUtils.memoryFilterKeyInfo(StringUtils.joinWith("-",filter,domain,"comment")));
        filterCrawlerRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(24,null));
        requestRecord.tagsCreator().bizTags().addCustomKV("comment_record_filter_info", JSON.toJSONString(filterCrawlerRecord));


        DevCrawlerController devCrawlerController = DevCrawlerController.builder()
                .triggerInfo(domain, domain + "_trigger", System.currentTimeMillis(), domain + "_job")
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain)) //内存队列
                .consoleResultPipeline() //控制台输t出
//                .fileResultPipeline("D:\\chance\\data\\toutiao\\toutiao_bk_test.json", false)
//                .requestRecord(flushNewsRecord)  //more job
                .requestRecord(requestRecord)  //more job
                .build();

        //是否采集评论
        devCrawlerController.getCrawlerJob().getScheduleTags().getCategoryTag().addLabelTag(CrawlerEnum.CrawlerDataType.comment.enumVal());
        devCrawlerController.getCrawlerJob().getScheduleTags().getCategoryTag().addKVTag(IS_FILTER_ARTICLE,true);
        devCrawlerController.start();
    }

    public static CrawlerRequestRecord flushNewsRecord(){
        FilterInfo turnPageFilterInfo = new FilterInfo();
        turnPageFilterInfo.setFilter(CrawlerEnum.CrawlerRecordFilter.count);
        turnPageFilterInfo.setCountTotalNum(100);
        turnPageFilterInfo.setCurCount(0);

        HttpConfig httpConfig = HttpConfig.me(domain);
        httpConfig.setDisableCookie(true);


        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, turnPageItem)
                .httpUrl("https://www.toutiao.com/")
                .releaseTime(System.currentTimeMillis())
//                .httpConfig(httpConfig)
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(7*24,null))//最大回溯时间范围
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(StringUtils.joinWith("-","filter",domain,site,"queue")))
                .turnPageFilterInfo(turnPageFilterInfo)
                .needWashed(false)
                .needParsed(true)
                .proxy(proxy)
                .build();
        Map<String,String> keysMap = new HashMap<>();
        keysMap.put("finance","3189399007");
//        keysMap.put("sports","3189398957");
        Map<String,Object> extras = new HashMap<>();
        extras.put("keysMap",keysMap);
        requestRecord.getHttpRequest().setExtras(extras);
        requestRecord.getHttpRequest().addExtra("ua_path","E:\\source\\ua.txt");

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        requestRecord.tagsCreator().bizTags().addSiteBiz(siteBiz);
        return requestRecord;
    }

    public static void articleRecordTest(){
        String url = "https://m.toutiaocdn.com/i1714211646243855/info/";
//        https://www.toutiao.com/w/a1714211646243855/?log_from=1081afef30f89_1634823521197
//        https://www.toutiao.com/w/a7001467045359616549/?log_from=1081afef30f89_1634823521197

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, turnPageItem)
                .httpUrl(url)
                .releaseTime(System.currentTimeMillis())
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(7*24,null))//最大回溯时间范围
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(StringUtils.joinWith("-","filter",domain,site,"queue")))
                .proxy(proxy)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.interaction)
                .build();

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        requestRecord.tagsCreator().bizTags().addSiteBiz(siteBiz);


        //添加评论去重信息
        CrawlerRecord filterCrawlerRecord = new CrawlerRecord();
        filterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.key);
        filterCrawlerRecord.addFilterInfo(FilterUtils.memoryFilterKeyInfo(StringUtils.joinWith("-",filter,domain,"comment")));
        filterCrawlerRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(24,null));
        requestRecord.tagsCreator().bizTags().addCustomKV("comment_record_filter_info", JSON.toJSONString(filterCrawlerRecord));

        DevCrawlerController devCrawlerController = DevCrawlerController.builder()
                .triggerInfo(domain, domain + "_trigger", System.currentTimeMillis(), domain + "_job")
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain)) //内存队列
                .consoleResultPipeline() //控制台输t出
//                .fileResultPipeline("D:\\chance\\data\\toutiao\\toutiao_bk_test.json", false)
//                .requestRecord(flushNewsRecord)  //more job
                .requestRecord(requestRecord)  //more job
                .supportRecord(userAgentRecord())
                .build();

        //是否采集评论
        devCrawlerController.getCrawlerJob().getScheduleTags().getCategoryTag().addLabelTag(CrawlerEnum.CrawlerDataType.comment.enumVal());
//        devCrawlerController.getCrawlerJob().getScheduleTags().getCategoryTag().addKVTag(IS_FILTER_ARTICLE,true);
        devCrawlerController.start();
    }


    public static CrawlerRequestRecord getKwRecord(String metaSite){
        //keyword record
        return CrawlerRequestRecord.builder()
                .startPageRequest(StringUtils.joinWith("-",domain, site),turnPageItem)
                .httpUrl("http://192.168.1.217:9599/v1/meta/"+domain+"/keys?site=" + metaSite)
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();
    }


    @Test
    public void test(){
        HttpConfig httpConfig = HttpConfig.me("test");
        httpConfig.setProxy(proxy);
        httpConfig.setUseProxy(true);

        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());

        for (int i = 0; i < 10; i++) {
            new Thread(new DownloadTest(httpConfig,downloader),"thread-" + i).start();
        }
    }
    static class DownloadTest implements Runnable{

        HttpConfig httpConfig;
        Downloader downloader;

        public DownloadTest(HttpConfig httpConfig, Downloader downloader) {
            this.httpConfig = httpConfig;
            this.downloader = downloader;
        }

        @Override
        public void run() {
            String url = "https://www.toutiao.com/api/pc/list/feed?channel_id=3189399007&max_behot_time=1626318701&category=pc_profile_channel";
            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setUrl(url);
            while (true){
                System.out.println(url);
                try {
                    HttpPage httpPage = downloader.download(httpRequest, httpConfig);
                    System.out.println(httpPage.getRawText());
                } catch (Exception e) {
                    break;
                }
            }
        }
    }


    private static CrawlerRequestRecord userAgentRecord(){
        CrawlerRequestRecord userAgentsCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("user_agents", turnPage)
                .httpUrl("http://fake-useragent.herokuapp.com/browsers/0.1.11")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();
        return userAgentsCrawlerRecord;
    }


    private static List<String> agentList = new ArrayList<>();

    static {
        agentList.add("Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1");
        agentList.add("Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/72.0.3626.101 Mobile/15E148 Safari/605.1");
        agentList.add("Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/15.0b13894 Mobile/16D57 Safari/605.1.15");
        agentList.add("Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/8.1.1 Mobile/16D57 Safari/605.1.15");
        agentList.add("Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/16D57");
        agentList.add("Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) OPiOS/16.0.14.122053 Mobile/16D57 Safari/9537.53");
        agentList.add("Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) OPT/2 Mobile/16D57");
        agentList.add("Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) OPiOS/12.0.5.3 Version/7.0 Mobile/16D57 Safari/9537.53");
        agentList.add("Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 EdgiOS/42.10.3 Mobile/16D57 Safari/605.1.15");
        agentList.add("Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/16D57 unknown BingWeb/6.9.8.1");
        agentList.add("Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 MQQBrowser/9.0.3 Mobile/16D57 Safari/604.1 MttCustomUA/2 QBWebViewType/1 WKType/1");
        agentList.add("Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/16D57 SearchCraft/3.4.1 (Baidu; P2 12.1.4)");
        agentList.add("Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_4 like Mac OS X; zh-CN) AppleWebKit/537.51.1 (KHTML, like Gecko) Mobile/16D57 UCBrowser/12.3.0.1138 Mobile AliApp(TUnionSDK/0.1.20.3)");
        agentList.add("Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_4 like Mac OS X; zh-cn) AppleWebKit/601.1.46 (KHTML, like Gecko) Mobile/16D57 Quark/3.0.6.926 Mobile");
        agentList.add("Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/16D57 MicroMessenger/7.0.3(0x17000321) NetType/WIFI Language/zh_CN");
        agentList.add("Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/16A366 QQ/7.8.8.420 V1_IPH_SQ_7.8.8_1_APP_A Pixel/1125 Core/WKWebView Device/Apple(iPhone X) NetType/4G QBWebViewType/1 WKType/1");
        agentList.add("Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12. Mobile/16D57 Safari/600.1.4 baidubrowser/4.14.1.11 (Baidu; P2 12.1.4)");
        agentList.add("Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/16D57 baiduboxapp/11.3.6.10 (Baidu; P2 12.1.4)");
        agentList.add("Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_4 like Mac OS X) AppleWebKit/606.4.5 (KHTML, like Gecko) Mobile/16D57 QHBrowser/317 QihooBrowser/4.0.10");
        agentList.add("Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/16D57 Mb2345Browser/5.2.1");
    }

    private static String getRandomUA() {
        return agentList.get(RandomUtils.nextInt(0, agentList.size() - 1));
    }
}
